{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "269bc007",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">DatasetDict</span><span style=\"font-weight: bold\">({</span>\n",
       "    train: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Dataset</span><span style=\"font-weight: bold\">({</span>\n",
       "        features: <span style=\"font-weight: bold\">[</span><span style=\"color: #008000; text-decoration-color: #008000\">'id'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'tokens'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'ner_tags'</span><span style=\"font-weight: bold\">]</span>,\n",
       "        num_rows: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20865</span>\n",
       "    <span style=\"font-weight: bold\">})</span>\n",
       "    validation: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Dataset</span><span style=\"font-weight: bold\">({</span>\n",
       "        features: <span style=\"font-weight: bold\">[</span><span style=\"color: #008000; text-decoration-color: #008000\">'id'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'tokens'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'ner_tags'</span><span style=\"font-weight: bold\">]</span>,\n",
       "        num_rows: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2319</span>\n",
       "    <span style=\"font-weight: bold\">})</span>\n",
       "    test: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Dataset</span><span style=\"font-weight: bold\">({</span>\n",
       "        features: <span style=\"font-weight: bold\">[</span><span style=\"color: #008000; text-decoration-color: #008000\">'id'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'tokens'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'ner_tags'</span><span style=\"font-weight: bold\">]</span>,\n",
       "        num_rows: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4637</span>\n",
       "    <span style=\"font-weight: bold\">})</span>\n",
       "<span style=\"font-weight: bold\">})</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mDatasetDict\u001b[0m\u001b[1m(\u001b[0m\u001b[1m{\u001b[0m\n",
       "    train: \u001b[1;35mDataset\u001b[0m\u001b[1m(\u001b[0m\u001b[1m{\u001b[0m\n",
       "        features: \u001b[1m[\u001b[0m\u001b[32m'id'\u001b[0m, \u001b[32m'tokens'\u001b[0m, \u001b[32m'ner_tags'\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        num_rows: \u001b[1;36m20865\u001b[0m\n",
       "    \u001b[1m}\u001b[0m\u001b[1m)\u001b[0m\n",
       "    validation: \u001b[1;35mDataset\u001b[0m\u001b[1m(\u001b[0m\u001b[1m{\u001b[0m\n",
       "        features: \u001b[1m[\u001b[0m\u001b[32m'id'\u001b[0m, \u001b[32m'tokens'\u001b[0m, \u001b[32m'ner_tags'\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        num_rows: \u001b[1;36m2319\u001b[0m\n",
       "    \u001b[1m}\u001b[0m\u001b[1m)\u001b[0m\n",
       "    test: \u001b[1;35mDataset\u001b[0m\u001b[1m(\u001b[0m\u001b[1m{\u001b[0m\n",
       "        features: \u001b[1m[\u001b[0m\u001b[32m'id'\u001b[0m, \u001b[32m'tokens'\u001b[0m, \u001b[32m'ner_tags'\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        num_rows: \u001b[1;36m4637\u001b[0m\n",
       "    \u001b[1m}\u001b[0m\u001b[1m)\u001b[0m\n",
       "\u001b[1m}\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from datasets import load_dataset,load_from_disk\n",
    "import os\n",
    "from rich import print\n",
    "import torch\n",
    "from transformers import AutoTokenizer,AutoModelForSequenceClassification,AutoModelForTokenClassification,BertForTokenClassification,BertForSequenceClassification,Trainer,TrainingArguments,DataCollatorForTokenClassification,DataCollatorWithPadding\n",
    "from torch.utils.data import Dataset,DataLoader\n",
    "\n",
    "\n",
    "import evaluate\n",
    "\n",
    "if os.getlogin() == 'caofei':\n",
    "    data_folder = r'C:\\Users\\caofei\\Desktop\\desktop link\\torch1\\hgface\\token_classification\\ner_data'\n",
    "    model_folder = r'D:\\Models\\chinese-macbert-base'\n",
    "else:\n",
    "    data_folder = r'C:\\Users\\COLORFUL\\Desktop\\AI_NLP\\hgface\\token_classification\\ner_data\\ner_data'\n",
    "    model_folder = r'C:\\Users\\COLORFUL\\.cache\\modelscope\\hub\\hfl\\chinese-macbert-base'\n",
    "ner_data = load_from_disk(data_folder)\n",
    "print(ner_data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "6fa9a869",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "random.randint(1,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2423317d",
   "metadata": {},
   "outputs": [],
   "source": [
    "[torch.randn(random.randint(1,10),).numpy().tolist() for _ in range(10)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "28b8d8b9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 2., 3.],\n",
       "        [1., 2., 1.]])"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.Tensor([[1,2,3],[1,2,1]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9d364709",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['海',\n",
       " '钓',\n",
       " '比',\n",
       " '赛',\n",
       " '地',\n",
       " '点',\n",
       " '在',\n",
       " '厦',\n",
       " '门',\n",
       " '与',\n",
       " '金',\n",
       " '门',\n",
       " '之',\n",
       " '间',\n",
       " '的',\n",
       " '海',\n",
       " '域',\n",
       " '。']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ner_data['train'][0]['tokens']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5e215786",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 0, 0, 0, 0, 0, 0, 5, 6, 0, 5, 6, 0, 0, 0, 0, 0, 0]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ner_data['train'][0]['ner_tags']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c5384ad9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['O', 'B-PER', 'I-PER', 'B-ORG', 'I-ORG', 'B-LOC', 'I-LOC']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_list=ner_data['train'].features['ner_tags'].feature.names\n",
    "label_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "217b9494",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\COLORFUL\\python_virtualenv\\torch-gt-2.0\\lib\\site-packages\\torch\\cuda\\__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.\n",
      "  import pynvml  # type: ignore[import]\n"
     ]
    }
   ],
   "source": [
    "# from modelscope import AutoTokenizer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fb897f21",
   "metadata": {},
   "outputs": [],
   "source": [
    "import evaluate\n",
    "from datasets import load_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d528b8ac",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForTokenClassification were not initialized from the model checkpoint at D:\\Models\\chinese-macbert-base and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at D:\\Models\\chinese-macbert-base and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "We strongly recommend passing in an `attention_mask` since your input_ids may be padded. See https://huggingface.co/docs/transformers/troubleshooting#incorrect-output-when-padding-tokens-arent-masked.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">torch.Size</span><span style=\"font-weight: bold\">([</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">10</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span><span style=\"font-weight: bold\">])</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mtorch.Size\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m5\u001b[0m, \u001b[1;36m10\u001b[0m, \u001b[1;36m2\u001b[0m\u001b[1m]\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">torch.Size</span><span style=\"font-weight: bold\">([</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span><span style=\"font-weight: bold\">])</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mtorch.Size\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m5\u001b[0m, \u001b[1;36m2\u001b[0m\u001b[1m]\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = AutoModelForTokenClassification.from_pretrained(model_folder)\n",
    "model2 = AutoModelForSequenceClassification.from_pretrained(model_folder)\n",
    "input = torch.randint(0,10,size=(5,10)).long()\n",
    "res = model(input_ids=input)\n",
    "res_logits = res.logits\n",
    "print(res_logits.size())\n",
    "res2 = model2(input_ids=input)\n",
    "res_logits2 = res2.logits\n",
    "print(res_logits2.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "81ec1730",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertTokenizerFast(name_or_path='D:\\Models\\chinese-macbert-base', vocab_size=21128, model_max_length=1000000000000000019884624838656, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'}, clean_up_tokenization_spaces=False, added_tokens_decoder={\n",
       "\t0: AddedToken(\"[PAD]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t100: AddedToken(\"[UNK]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t101: AddedToken(\"[CLS]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t102: AddedToken(\"[SEP]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t103: AddedToken(\"[MASK]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "}\n",
       ")"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# tokenizer = AutoTokenizer.from_pretrained(\"hfl/chinese-macbert-base\")\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_folder)\n",
    "tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "dcea37d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "type(tokenizer)\n",
    "from transformers.models.bert.tokenization_bert_fast import BertTokenizerFast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "9397f4ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['id', 'tokens', 'ner_tags'],\n",
       "    num_rows: 4637\n",
       "})"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ner_data['test']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "9fc7431b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span>\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'input_ids'</span>: <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">101</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">10673</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">12865</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">12921</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">8181</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">162</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">10716</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">8118</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">102</span><span style=\"font-weight: bold\">]</span>,\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'token_type_ids'</span>: <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span><span style=\"font-weight: bold\">]</span>,\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'attention_mask'</span>: <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span><span style=\"font-weight: bold\">]</span>\n",
       "<span style=\"font-weight: bold\">}</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m{\u001b[0m\n",
       "    \u001b[32m'input_ids'\u001b[0m: \u001b[1m[\u001b[0m\u001b[1;36m101\u001b[0m, \u001b[1;36m10673\u001b[0m, \u001b[1;36m12865\u001b[0m, \u001b[1;36m12921\u001b[0m, \u001b[1;36m8181\u001b[0m, \u001b[1;36m162\u001b[0m, \u001b[1;36m10716\u001b[0m, \u001b[1;36m8118\u001b[0m, \u001b[1;36m102\u001b[0m\u001b[1m]\u001b[0m,\n",
       "    \u001b[32m'token_type_ids'\u001b[0m: \u001b[1m[\u001b[0m\u001b[1;36m0\u001b[0m, \u001b[1;36m0\u001b[0m, \u001b[1;36m0\u001b[0m, \u001b[1;36m0\u001b[0m, \u001b[1;36m0\u001b[0m, \u001b[1;36m0\u001b[0m, \u001b[1;36m0\u001b[0m, \u001b[1;36m0\u001b[0m, \u001b[1;36m0\u001b[0m\u001b[1m]\u001b[0m,\n",
       "    \u001b[32m'attention_mask'\u001b[0m: \u001b[1m[\u001b[0m\u001b[1;36m1\u001b[0m, \u001b[1;36m1\u001b[0m, \u001b[1;36m1\u001b[0m, \u001b[1;36m1\u001b[0m, \u001b[1;36m1\u001b[0m, \u001b[1;36m1\u001b[0m, \u001b[1;36m1\u001b[0m, \u001b[1;36m1\u001b[0m, \u001b[1;36m1\u001b[0m\u001b[1m]\u001b[0m\n",
       "\u001b[1m}\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[None, 0, 0, 0, 0, 1, 1, 1, None]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "st = tokenizer('interesting things')\n",
    "print(st)\n",
    "\n",
    "st.word_ids()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "2ddccf09",
   "metadata": {
    "vscode": {
     "languageId": "markdown"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<function torch._VariableFunctionsClass.argmax>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.argmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c3e83c1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 计算交叉熵损失\n",
    "import torch\n",
    "import torch.nn as nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "079eacff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(2.6821)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss = nn.CrossEntropyLoss()\n",
    "\n",
    "init_input = torch.randn(5,10)\n",
    "# <!-- display(init_input.size())\n",
    "# input = torch.randn(5,10,768) -->\n",
    "input = torch.randn(5,10,768)\n",
    "labels = torch.randint(0,10,size=(5,))\n",
    "loss(init_input,labels)\n",
    "# torch.argmax(torch.softmax(input,dim=-1),dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "84e41500",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(7.0798)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss = nn.CrossEntropyLoss()\n",
    "\n",
    "init_input = torch.randn(5,10)\n",
    "# <!-- display(init_input.size())\n",
    "# input = torch.randn(5,10,768) -->\n",
    "input = torch.randn(5,10,768)\n",
    "labels = torch.randint(0,10,size=(5,10))\n",
    "\n",
    "loss(input.reshape(50,768),labels.reshape(50,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "88591bad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 0, 0, 0, 0, 0, 0, 5, 6, 0, 5, 6, 0, 0, 0, 0, 0, 0]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "['海',\n",
       " '钓',\n",
       " '比',\n",
       " '赛',\n",
       " '地',\n",
       " '点',\n",
       " '在',\n",
       " '厦',\n",
       " '门',\n",
       " '与',\n",
       " '金',\n",
       " '门',\n",
       " '之',\n",
       " '间',\n",
       " '的',\n",
       " '海',\n",
       " '域',\n",
       " '。']"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 输入  \n",
    "display(ner_data['train'][0]['ner_tags'])\n",
    "display(ner_data['train'][0]['tokens'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "23b33b6f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1., 2., 3.])"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.Tensor([1,2,3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "a11b2ec3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def map_function(data):\n",
    "    output = tokenizer(data['tokens'],max_length=128,truncation=True,is_split_into_words=True,return_tensors='pt')\n",
    "    # output['labels'] = data['ner_tags']\n",
    "    labels = []\n",
    "    # 需要拼凑起来\n",
    "    tags = data[\"ner_tags\"]\n",
    "    for k in output.word_ids(): #[None,0,0,1,None]\n",
    "        if k is None:\n",
    "            labels.append(-100)\n",
    "        else:\n",
    "            labels.append(tags[k])\n",
    "        \n",
    "    \n",
    "    # for i, label in enumerate(data[\"ner_tags\"]):\n",
    "    #     # # i=0 1 2  label\n",
    "    #     # word_ids = output.word_ids(batch_index=i)\n",
    "        \n",
    "    \n",
    "    output['labels'] = torch.Tensor(labels)\n",
    "    \n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "3553f42a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'train': ['id', 'tokens', 'ner_tags'],\n",
       " 'validation': ['id', 'tokens', 'ner_tags'],\n",
       " 'test': ['id', 'tokens', 'ner_tags']}"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ner_data.column_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "d287dd84",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1c62232032b4276849cb191cd8eb9b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/20865 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6bda60c28a524b5b8b3e47abd447213a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/2319 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a56f4b41f5214272ab44d1c37f1114a1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/4637 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['input_ids', 'token_type_ids', 'attention_mask', 'labels'],\n",
       "        num_rows: 20865\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['input_ids', 'token_type_ids', 'attention_mask', 'labels'],\n",
       "        num_rows: 2319\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['input_ids', 'token_type_ids', 'attention_mask', 'labels'],\n",
       "        num_rows: 4637\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ner_data_tokiner = ner_data.map(map_function,remove_columns=ner_data.column_names['test'])\n",
    "ner_data_tokiner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1926dfe7",
   "metadata": {},
   "outputs": [],
   "source": [
    "ner_data_tokiner['train'][1]['input_ids']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "a0cd65cb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 101, 6821, 2429,  898, 2255,  988, 3717, 4638, 1300, 4289, 7667, 4507,\n",
       "         1744, 1079,  671, 3837, 4638, 6392, 6369, 2360,  712, 2898, 6392, 6369,\n",
       "         8024, 3146,  702, 2456, 5029, 5408, 5125, 5401, 5445, 2612, 2131,  511,\n",
       "          102]])"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.Tensor(ner_data_tokiner['train'][1]['input_ids']).long()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "d1bb0d0e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['O', 'B-PER', 'I-PER', 'B-ORG', 'I-ORG', 'B-LOC', 'I-LOC']"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "63321f8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "ner_data_tokiner.remove_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fbb95d6e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['input_ids', 'token_type_ids', 'attention_mask', 'labels'])"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ner_data_tokiner['train'][1].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "12d1141e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'torch.LongTensor'"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d = ner_data_tokiner['train'][1]\n",
    "torch.Tensor(d['input_ids']).long().type()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "c556bd5d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "37"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(d['input_ids'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'ner_data_tokiner' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[72], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m d \u001b[38;5;241m=\u001b[39m \u001b[43mner_data_tokiner\u001b[49m[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtrain\u001b[39m\u001b[38;5;124m'\u001b[39m][\u001b[38;5;241m1\u001b[39m]\n\u001b[0;32m      2\u001b[0m model \u001b[38;5;241m=\u001b[39m AutoModelForTokenClassification\u001b[38;5;241m.\u001b[39mfrom_pretrained(model_folder,num_labels\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mlen\u001b[39m(label_list))\n\u001b[0;32m      3\u001b[0m res \u001b[38;5;241m=\u001b[39m model(input_ids\u001b[38;5;241m=\u001b[39mtorch\u001b[38;5;241m.\u001b[39mTensor(d[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124minput_ids\u001b[39m\u001b[38;5;124m'\u001b[39m])\u001b[38;5;241m.\u001b[39mlong(),token_type_ids\u001b[38;5;241m=\u001b[39mtorch\u001b[38;5;241m.\u001b[39mTensor(d[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtoken_type_ids\u001b[39m\u001b[38;5;124m'\u001b[39m])\u001b[38;5;241m.\u001b[39mlong(),attention_mask\u001b[38;5;241m=\u001b[39mtorch\u001b[38;5;241m.\u001b[39mTensor(d[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124minput_ids\u001b[39m\u001b[38;5;124m'\u001b[39m])\u001b[38;5;241m.\u001b[39mlong(),labels\u001b[38;5;241m=\u001b[39mtorch\u001b[38;5;241m.\u001b[39mTensor(d[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mlabels\u001b[39m\u001b[38;5;124m'\u001b[39m])\u001b[38;5;241m.\u001b[39mlong())\n",
      "\u001b[1;31mNameError\u001b[0m: name 'ner_data_tokiner' is not defined"
     ]
    }
   ],
   "source": [
    "\n",
    "d = ner_data_tokiner['train'][1]\n",
    "model = AutoModelForTokenClassification.from_pretrained(model_folder,num_labels=len(label_list))\n",
    "res = model(input_ids=torch.Tensor(d['input_ids']).long(),token_type_ids=torch.Tensor(d['token_type_ids']).long(),attention_mask=torch.Tensor(d['input_ids']).long(),labels=torch.Tensor(d['labels']).long())\n",
    "# res.logits.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "6305a20f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.config.num_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5b9dd96",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "a91e2249",
   "metadata": {},
   "outputs": [],
   "source": [
    "args = TrainingArguments(\n",
    "    \n",
    "    output_dir=r'C:\\Users\\caofei\\Desktop\\desktop link\\torch1\\hgface\\token_classification\\models',\n",
    "    per_device_train_batch_size = 64,\n",
    "    per_device_eval_batch_size= 64,\n",
    "    save_strategy=\"epoch\",\n",
    "    metric_for_best_model=\"f1\",\n",
    "    logging_steps=10,\n",
    "    num_train_epochs=3,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "c0092f2c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\caofei\\AppData\\Local\\Temp\\ipykernel_4216\\1054866278.py:1: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
      "  trainer = Trainer(\n"
     ]
    }
   ],
   "source": [
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=args,\n",
    "    train_dataset=ner_data_tokiner['train'],\n",
    "    eval_dataset=ner_data_tokiner['validation'],\n",
    "    tokenizer = tokenizer,\n",
    "    data_collator=DataCollatorForTokenClassification(tokenizer=tokenizer)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">swanlab</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">:</span> Using SwanLab to track your experiments. Please refer to <span style=\"color: #808000; text-decoration-color: #808000\">https://docs.swanlab.cn</span> for more information.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;34mswanlab\u001b[0m\u001b[1;39m:\u001b[0m Using SwanLab to track your experiments. Please refer to \u001b[33mhttps://docs.swanlab.cn\u001b[0m for more information.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">swanlab</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">:</span> <span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span><span style=\"font-weight: bold\">)</span> Create a SwanLab account.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;34mswanlab\u001b[0m\u001b[1;39m:\u001b[0m \u001b[1m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1m)\u001b[0m Create a SwanLab account.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">swanlab</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">:</span> <span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span><span style=\"font-weight: bold\">)</span> Use an existing SwanLab account.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;34mswanlab\u001b[0m\u001b[1;39m:\u001b[0m \u001b[1m(\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1m)\u001b[0m Use an existing SwanLab account.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">swanlab</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">:</span> <span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span><span style=\"font-weight: bold\">)</span> Don't visualize my results.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;34mswanlab\u001b[0m\u001b[1;39m:\u001b[0m \u001b[1m(\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1m)\u001b[0m Don't visualize my results.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #000080; text-decoration-color: #000080; font-weight: bold\">swanlab</span><span style=\"color: #000000; text-decoration-color: #000000; font-weight: bold\">:</span> Enter your choice: \n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;34mswanlab\u001b[0m\u001b[1;39m:\u001b[0m Enter your choice: \n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "926b4e9d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "97a181c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "for d in ner_data_tokiner:\n",
    "    assert len(d['labels']) == len(d['input_ids'])\n",
    "    \n",
    "    # ner_data_tokiner[0]['labels']\n",
    "    # ner_data_tokiner[0]['input_ids']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a96f97c1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'我们变而以书会友，以书结缘，把欧美、港台流行的食品类图谱、画册、工具书汇集一堂。'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'input_ids': [101, 2769, 812, 1359, 5445, 809, 741, 833, 1351, 8024, 809, 741, 5310, 5357, 8024, 2828, 3616, 5401, 510, 3949, 1378, 3837, 6121, 4638, 7608, 1501, 5102, 1745, 6480, 510, 4514, 1085, 510, 2339, 1072, 741, 3726, 7415, 671, 1828, 511, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[None,\n",
       " 0,\n",
       " 1,\n",
       " 2,\n",
       " 3,\n",
       " 4,\n",
       " 5,\n",
       " 6,\n",
       " 7,\n",
       " 8,\n",
       " 9,\n",
       " 10,\n",
       " 11,\n",
       " 12,\n",
       " 13,\n",
       " 14,\n",
       " 15,\n",
       " 16,\n",
       " 17,\n",
       " 18,\n",
       " 19,\n",
       " 20,\n",
       " 21,\n",
       " 22,\n",
       " 23,\n",
       " 24,\n",
       " 25,\n",
       " 26,\n",
       " 27,\n",
       " 28,\n",
       " 29,\n",
       " 30,\n",
       " 31,\n",
       " 32,\n",
       " 33,\n",
       " 34,\n",
       " 35,\n",
       " 36,\n",
       " 37,\n",
       " 38,\n",
       " 39,\n",
       " None]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dd2 = ner_data['test'][0]\n",
    "dd = dd2['tokens']\n",
    "\n",
    "display(''.join(dd))\n",
    "res = tokenizer(dd,is_split_into_words=True)\n",
    "display(res)\n",
    "display(res.word_ids(batch_index=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed5da692",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "acea881c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "5a06d11b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "07a5e6f68ea6419db3eecfef6de8675f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/4637 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "ner_data_tokiner = ner_data['test'].map(map_function)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d8f4bc0a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "id": "7a4152c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['id', 'tokens', 'ner_tags'],\n",
       "        num_rows: 20865\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['id', 'tokens', 'ner_tags'],\n",
       "        num_rows: 2319\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['id', 'tokens', 'ner_tags'],\n",
       "        num_rows: 4637\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ner_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "9aec15d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">40</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m40\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">42</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;36m42\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "s = ner_data_tokiner[0]\n",
    "print(len(s['labels']))\n",
    "\n",
    "print(len(s['input_ids']))\n",
    "\n",
    "# print(tokenizer.decode(s['input_ids']))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f51ab89b",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(s['labels'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "cd312a29",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'我 们 变 而 以 书 会 友 ， 以 书 结 缘 ， 把 欧 美 、 港 台 流 行 的 食 品 类 图 谱 、 画 册 、 工 具 书 汇 集 一 堂 。'"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.decode(s['input_ids'],skip_special_tokens=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer.vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "f929b0c6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "39"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ss = '我 们 变 而 以 书 会 友 ， 以 书 结 缘 ， 把 欧 美 、 港 台 流 行 的 食 品 类 图 谱 、 画 册 、 工 具 书 汇 集 一 堂'.replace(' ','')\n",
    "ss\n",
    "\n",
    "len(ss)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "39"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tokenizer(ss,add_special_tokens=False)['input_ids'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19781eed",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a347553e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "7bc91a37",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "res = tokenizer(\"我爱中国人\")\n",
    "\n",
    "res2 = tokenizer(\"interesting word\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "8d3ebfdd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span>\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'input_ids'</span>: <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">101</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2769</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4263</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">704</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1744</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">782</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">102</span><span style=\"font-weight: bold\">]</span>,\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'token_type_ids'</span>: <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span><span style=\"font-weight: bold\">]</span>,\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'attention_mask'</span>: <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span><span style=\"font-weight: bold\">]</span>\n",
       "<span style=\"font-weight: bold\">}</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m{\u001b[0m\n",
       "    \u001b[32m'input_ids'\u001b[0m: \u001b[1m[\u001b[0m\u001b[1;36m101\u001b[0m, \u001b[1;36m2769\u001b[0m, \u001b[1;36m4263\u001b[0m, \u001b[1;36m704\u001b[0m, \u001b[1;36m1744\u001b[0m, \u001b[1;36m782\u001b[0m, \u001b[1;36m102\u001b[0m\u001b[1m]\u001b[0m,\n",
       "    \u001b[32m'token_type_ids'\u001b[0m: \u001b[1m[\u001b[0m\u001b[1;36m0\u001b[0m, \u001b[1;36m0\u001b[0m, \u001b[1;36m0\u001b[0m, \u001b[1;36m0\u001b[0m, \u001b[1;36m0\u001b[0m, \u001b[1;36m0\u001b[0m, \u001b[1;36m0\u001b[0m\u001b[1m]\u001b[0m,\n",
       "    \u001b[32m'attention_mask'\u001b[0m: \u001b[1m[\u001b[0m\u001b[1;36m1\u001b[0m, \u001b[1;36m1\u001b[0m, \u001b[1;36m1\u001b[0m, \u001b[1;36m1\u001b[0m, \u001b[1;36m1\u001b[0m, \u001b[1;36m1\u001b[0m, \u001b[1;36m1\u001b[0m\u001b[1m]\u001b[0m\n",
       "\u001b[1m}\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[None, 0, 1, 2, 3, 4, None]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(res)\n",
    "res.word_ids()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "15affbe9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span>\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'input_ids'</span>: <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">101</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">10673</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">12865</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">12921</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">8181</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">8681</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">102</span><span style=\"font-weight: bold\">]</span>,\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'token_type_ids'</span>: <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span><span style=\"font-weight: bold\">]</span>,\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'attention_mask'</span>: <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span><span style=\"font-weight: bold\">]</span>\n",
       "<span style=\"font-weight: bold\">}</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m{\u001b[0m\n",
       "    \u001b[32m'input_ids'\u001b[0m: \u001b[1m[\u001b[0m\u001b[1;36m101\u001b[0m, \u001b[1;36m10673\u001b[0m, \u001b[1;36m12865\u001b[0m, \u001b[1;36m12921\u001b[0m, \u001b[1;36m8181\u001b[0m, \u001b[1;36m8681\u001b[0m, \u001b[1;36m102\u001b[0m\u001b[1m]\u001b[0m,\n",
       "    \u001b[32m'token_type_ids'\u001b[0m: \u001b[1m[\u001b[0m\u001b[1;36m0\u001b[0m, \u001b[1;36m0\u001b[0m, \u001b[1;36m0\u001b[0m, \u001b[1;36m0\u001b[0m, \u001b[1;36m0\u001b[0m, \u001b[1;36m0\u001b[0m, \u001b[1;36m0\u001b[0m\u001b[1m]\u001b[0m,\n",
       "    \u001b[32m'attention_mask'\u001b[0m: \u001b[1m[\u001b[0m\u001b[1;36m1\u001b[0m, \u001b[1;36m1\u001b[0m, \u001b[1;36m1\u001b[0m, \u001b[1;36m1\u001b[0m, \u001b[1;36m1\u001b[0m, \u001b[1;36m1\u001b[0m, \u001b[1;36m1\u001b[0m\u001b[1m]\u001b[0m\n",
       "\u001b[1m}\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[None, 0, 0, 0, 0, 1, None]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(res2)\n",
    "res2.word_ids()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "eb9751c6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['O', 'B-PER', 'I-PER', 'B-ORG', 'I-ORG', 'B-LOC', 'I-LOC']"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "94c54697",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['海',\n",
       " '钓',\n",
       " '比',\n",
       " '赛',\n",
       " '地',\n",
       " '点',\n",
       " '在',\n",
       " '厦',\n",
       " '门',\n",
       " '与',\n",
       " '金',\n",
       " '门',\n",
       " '之',\n",
       " '间',\n",
       " '的',\n",
       " '海',\n",
       " '域',\n",
       " '。']"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[0, 0, 0, 0, 0, 0, 0, 5, 6, 0, 5, 6, 0, 0, 0, 0, 0, 0]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 规范化数据\n",
    "\n",
    "d = ner_data['train'][0]['tokens']\n",
    "label = ner_data['train'][0]['ner_tags']\n",
    "display(d,label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "3675cf39",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 评估指标\n",
    "\n",
    "# 精确率\n",
    "precision = r\"C:\\Users\\caofei\\Desktop\\evaluate-main\\metrics\\precision\\precision.py\"\n",
    "metric_precision = evaluate.load(precision)\n",
    "\n",
    "# 召回率\n",
    "recall = r\"C:\\Users\\caofei\\Desktop\\evaluate-main\\metrics\\recall\\recall.py\"\n",
    "metric_recall = evaluate.load(recall)\n",
    "\n",
    "\n",
    "# ner 相关指标\n",
    "seqeval = r\"C:\\Users\\caofei\\Desktop\\evaluate-main\\metrics\\seqeval\\seqeval.py\"\n",
    "metric_seqeval = evaluate.load(seqeval)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "abdd86df",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'precision': 0.6666666666666666}"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metric_precision.compute(references=[1,1,0],predictions=[1,1,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "60649755",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'recall': 1.0}"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 实际上是真实的,预测是真实的\n",
    "metric_recall.compute(references=[0,1,1],predictions=[0,1,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "5c2efee2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">整体精确率: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.5000</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "整体精确率: \u001b[1;36m0.5000\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">整体召回率: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.5000</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "整体召回率: \u001b[1;36m0.5000\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">PER实体的F1: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.0000</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "PER实体的F1: \u001b[1;36m1.0000\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# ner的指标\n",
    "import evaluate\n",
    "\n",
    "# 加载seqeval指标（本质上还是调用seqeval库）\n",
    "# seqeval = evaluate.load(\"seqeval\")\n",
    "\n",
    "# y_true = [\n",
    "#     ['O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'O'],\n",
    "#     ['B-PER', 'I-PER', 'O']\n",
    "# ]\n",
    "# y_pred = [\n",
    "#     ['O', 'O', 'O', 'B-MISC', 'I-MISC', 'O'],\n",
    "#     ['B-PER', 'I-PER', 'O']\n",
    "# ]\n",
    "\n",
    "# # 计算指标\n",
    "# results = metric_seqeval.compute(predictions=y_pred, references=y_true)\n",
    "\n",
    "# print(f\"整体精确率: {results['overall_precision']:.4f}\")  # 0.5\n",
    "# print(f\"整体召回率: {results['overall_recall']:.4f}\")     # 0.5\n",
    "# print(f\"PER实体的F1: {results['PER']['f1']:.4f}\")         # 1.0（PER实体全部正确）  y_true = [\n",
    "\n",
    "\n",
    "y_true = [\n",
    "    ['O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'O'],\n",
    "    ['B-PER','I-PER','O']\n",
    "]\n",
    "y_pred = [\n",
    "    ['O', 'O', 'O', 'B-MISC', 'I-MISC', 'O'],\n",
    "    ['B-PER','I-PER','O']\n",
    "]\n",
    "\n",
    "# 计算指标\n",
    "results = metric_seqeval.compute(predictions=y_pred, references=y_true)\n",
    "\n",
    "print(f\"整体精确率: {results['overall_precision']:.4f}\")  # 0.5\n",
    "print(f\"整体召回率: {results['overall_recall']:.4f}\")     # 0.5\n",
    "print(f\"PER实体的F1: {results['PER']['f1']:.4f}\")         # 1.0（PER实体全部正确）  y_true = [\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7db2af4b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch_py10",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
